#for(i in 1:length(df_sample$Body))
#{
#  str<-paste("./corpus_dir/body",i,sep="")
#  write(df_sample$Body[i], file=str)
#}

#txt <- system.file("texts", "txt", package = "tm")
#mycorpus <- PCorpus(DirSource("."),readerControl = list(language = "en"))

library(tm)
library(SnowballC)
library(rpart)

df_sample<-read.csv("./train_sample.csv",stringsAsFactors=F)

v.text<-paste(df_sample$Title, df_sample$Body, sep=" ")

v.text<-tolower(v.text)
v.text<- gsub("<code>.*</code>"," ",v.text)
v.text<- gsub("<p>"," ",v.text, fixed=T)
v.text<- gsub("</p>"," ",v.text, fixed=T)
v.text<- gsub("\n"," ",v.text, fixed=T)
v.text<- gsub("<strong>"," ",v.text)
v.text<- gsub("<p>"," ",v.text)
v.text<- gsub("<strong>"," ",v.text)
v.text<- gsub("</strong>"," ",v.text)
v.text<- gsub("<pre>"," ",v.text)
v.text<- gsub("</pre>"," ",v.text)
v.text<- gsub("\\(.*\\)","",v.text)
v.text<- gsub("c#","csharp",v.text, fixed=T)               ##So c# and c++ will be recognized as term
v.text<- gsub("f#","fsharp",v.text, fixed=T)
v.text<- gsub("c++","cplusplus",v.text, fixed=T)
v.text<-gsub("(?!')[[:punct:]]", " ", v.text, perl=TRUE)


text.corpus<-tm_map(Corpus(VectorSource(v.text)),tolower)

mystops<-c(
 "a","about","above","after","again","against","all","am",
"an","and","any","appreciate","aren't","are","as","at",
"be","because","been","before","being","below","between",
"both","but","by","can't","cannot","couldn't","could",
"didn't","did","do","doesn't","does","doing","don't",
"down","during","each","few","for","from","further",
"hadn't","had","hasn't","has","haven't","have","having",
"he'd","he'll","he's","he","help","her","here's","here",
"hers","herself","him","himself","his","how's","how",
"i'd","i'll","i'm","i've","i","if","in","into","isn't",
"is","it's","it","its","itself","let's","me","more",
"most","mustn't","my","myself","no","nor","not","of",
"off","on","once","only","or","other","ought","our",
"ours ","ourselves","out","over","own","same","shan't",
"she'd","she'll","she's","shouldn't","should","she","so",
"some","such","than","thank","thanks","that's","that","the",
"theirs","their","them","themselves","then","there's","there",
"these","they'd","they'll","they're","they've","they",
"this","those","through","to","too","under","until","up",
"very","wasn't","was","we'd","we'll","we're","we've",
"we","were","weren't","what's","what","when's","when",
"where's","where","which","while","who's","who","whom",
"why's","why","with","won't","wouldn't","would","you'd",
"you'll","you're","you've","you","your","yours","yourself",
"your","application", "can", "code", "create", "data", "error",
"find", "following", "get", "just", "know","like", "need", "pre",
"use", "using", "want", "way", "will","now", "one", "problem",
 "something", "sure", "trying", "work", "right", "run", "running",
 "see", "seems", "set", "show", "similar", "simple", "since",
 "possible",  "value","method","also","app","time","works" , "please"
)

text.corpus<-tm_map(text.corpus, removeWords, mystops )
text.corpus<-tm_map(text.corpus, removeWords, stopwords("english"))
text.corpus<-tm_map(text.corpus, stripWhitespace)

##USE SEPARATE VARIABLES FOR TERM FREQUENCY IN TITLE VERSUS BODY? CODE TOO?
##DOES > 25 THRESHOLD HELP OR NOT?

dtm <- removeSparseTerms(DocumentTermMatrix(text.corpus), 0.999)

num.tags<-500
tag.freq<-table(unlist(strsplit(df_sample$Tags," ")))                       ##Get tag names and their frequency on the training data
tag.freq<-tag.freq[rev(order(tag.freq))]
tag.names<-names(tag.freq)

 sens<-rep(NA, num.tags)
spec<-rep(NA,num.tags)
f1stat<-rep(NA,num.tags)


sensitivity<-function(thr, p, resp){
  pred<-(1:length(p))[p > thr]
  act<-(1:length(resp))[resp>0]
  if (length(pred)>0) length(intersect(pred, act))/length(pred)  else 0
}

specificity<-function(thr, p,  resp){
  pred<-(1:length(p))[p > thr]
  act<-(1:length(resp))[resp>0]
  if (length(act)>0) length(intersect(pred, act))/length(act)  else 0
}

tag.names[7]<-"c\\+\\+"


for(i in 1:num.tags){

search.string<-paste("^",tag.names[i]," | ",tag.names[i]," | ",tag.names[i],"$",sep="")
act<-grep(search.string, df_sample$Tags)
dtm.tag<-dtm[act,]

freq.all<-apply(dtm, 2, sum)/nrow(dtm)
freq.tag<-apply(dtm.tag, 2, sum)/nrow(dtm.tag)
ind<-(apply(dtm, 2, sum) > 5)
freq.all<-freq.all[ind]
freq.tag<-freq.tag[ind]
freq.diff<-freq.all - freq.tag
freq.all<-freq.all[order(freq.diff)]
freq.tag<-freq.tag[order(freq.diff)]
freq.diff<-freq.diff[order(freq.diff)]
u1<-row.names(cbind(freq.all, freq.tag, freq.diff))
L<-length(freq.diff)
u2<-c(u1[1:20], u1[(L-19):L])
v<-unlist(dtm$dimnames[2])
ind<-sapply(u2, function(x) (1:length(v))[v==x])
response<-rep(0, length(df_sample$Tags))
response[act]<-1


df<-data.frame(as.matrix(dtm)[,ind],response)
#m<-rpart(response ~ ., data=df)
m<-glm(response~.,data=df, family=binomial)
probs<-predict(m,type="resp")
x1<-sapply(seq(.01,1,.01), sensitivity, probs, df$response)
x2<-sapply(seq(.01,1,.01), specificity, probs, df$response)

F<-2*x1*x2/(x1+x2)
F[is.na(F)]<-0
max(F)
j<-which.max(F)
cat(i, tag.names[i], x1[j], x2[j], F[j], "\n")
sens[i]<-x1[j]
spec[i]<-x2[j]
f1stat[i]<-F[j]
}

model.perf.info<-data.frame(tag=tag.names,sens=sens, spec=spec, f1stat=f1stat)




sens[i]<-sensitivity(probs, df$response, threshold)
spec[i]<-specificity(probs, df$response, threshold)
